import pandas
from hmmlearn import hmm
import numpy
#采用SBME分词  转化为数值型分别是0 1 2 3
words=[]
states=[]
chars=set()
with open("RenMinData.txt_utf8",mode="r",encoding="utf-8") as f:
    lines=f.readlines()
for line in lines:
    new_line=line.strip().split(" ")
    s=[]
    for w in new_line:
        if len(w)==1:
            s=s+[0]
            chars.add(w)
        else:
            s=s+[1]+[2]*(len(w)-2)+[3]
            for c in w:
                chars.add(c)
    words.append("".join(new_line))
    states.append(s)
chars=list(chars)
chars.sort()
#将观测结果转化为数值型
d=dict([(chars[i],i) for i in range(len(chars))])
new_words=[]
for w in words:
    word=[]
    for c in w:
        word.append(d[c])
    new_words.append(word)

# print(states)
# print(new_words)
#标点符号也要考虑进去
#进行统计
#计算初始概率矩阵
pi=numpy.zeros(4)
A=numpy.zeros((4,4))
B=numpy.zeros((4,len(chars)))
# df_states=pandas.DataFrame(states)
# df_states.fillna(-1,inplace=True)
# df_states=df_states.astype(int)
# p=df_states[0].value_counts(normalize=True,ascending=True)
# for i in range(4):
#     pi[i]=p.get(i,0)
# #计算状态转移矩阵A
# for i in range(4):
#     for j in range(4):
#         for column in  df_states.columns[:-1]:
#             A[i,j]+=((df_states[column]==i)&(df_states[column+1]==j)).sum()
# A=A/numpy.sum(A,axis=1).reshape(-1,1)
# s=numpy.concatenate(states)
# w=numpy.concatenate(new_words)
# #计算状态转移矩阵B  直接将所有数据合并在一起进行计算
# for i in range(4):
#     t=(s==i)
#     for j in range(len(chars)):
#         B[i,j]=numpy.sum(w[t]==j)
# B=B/numpy.sum(B,axis=1).reshape(-1,1)
#直接扫描数据集，而不是根据条件扫描
for i in range(len(states)):
    for j in range(len(states[i])):
        if j==0:
            pi[states[i][j]]+=1
            B[states[i][j],new_words[i][j]]+=1
        else:
            A[states[i][j-1],states[i][j]]+=1
            B[states[i][j], new_words[i][j]] += 1
pi=pi/pi.sum()
A=A/numpy.sum(A,axis=1).reshape((-1,1))
B=B/numpy.sum(B,axis=1).reshape((-1,1))
model=hmm.CategoricalHMM(n_components=4,n_features=len(chars))
model.transmat_=A
model.emissionprob_=B
model.startprob_=pi
